import requests
from lxml import etree

url = 'https://movie.douban.com/subject/1292052/comments'

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36',
}

proxies = {
    'https': 'https://113.206.176.55:13932',
    # 'http': 'http://113.140.188.27:4256'
}

r = requests.get(url=url, headers=headers)
print(type(r.text))

html_etree = etree.HTML(r.text)
# html_rating = html_etree.xpath('//div[@class="comment"]//span[@class="comment-info"]/span[2]/@class')
# print(html_rating)
# html_comment = html_etree.xpath('//div[@class="comment"]//span[@class="short"]/text()')
# print(html_comment)

html_all = html_etree.xpath('//div[@class="comment"]')
html_list = list()
for html in html_all:
    html_dic = dict()
    a = html.xpath('//span[@class="comment-info"]/span[2]/@class')
    if a:
        html_dic['rating'] = a
    else:
        html_dic['rating'] = None
    b = html.xpath('//div[@class="comment"]//span[@class="short"]/text()')
    if b:
        html_dic['comment'] = b
    else:
        html_dic['comment'] = None
    html_list.append(html_dic)
print(html_list)
print(len(html_list))
